"""
学习模块
主要实现三种学习机制：
1. *进化学习
2. *模仿学习
3. 强化学习
"""
import random
import time
import math
from math import ceil
from typing import List
import numpy as np
import pandas as pd
from collections import deque, defaultdict

# 进化学习（目前主要通过遗传算法来实现）
class EvolutionaryLearning:
    class Individual:
        """个体类"""

        def __init__(self, id, chromosome, fitness):
            """
            Args:
                id: 个体唯一的id号
                chromosome: 个体的染色体编号
            """
            self.id = id
            self.chromosome = chromosome
            self.fitness = fitness

    def __init__(self, population_capacity):
        """
        Args:
            population_capacity: 种群最大容纳量
        """
        self.population_capacity = population_capacity

        self.group = []  # 初始种群
        self.update_group = []
        self.update_group_id = 0
        self.generation = 0  # 遗传进行的代数
        self.update_info = []

    def add_to_group(self, individual_id, individual_chromosome, individual_fitness):
        """
        往种群中添加个体
        Args:
            individual_id: 个体的id
            individual_chromosome: 个体的染色体，用列表形式表示
            individual_fitness: 个体的适应度
        """
        if len(self.group) <= self.population_capacity:
            individual = self.Individual(
                id=individual_id,
                chromosome=individual_chromosome,
                fitness=individual_fitness
            )
            self.group.append(individual)
            print("len:", len(self.group))
        else:
            print("个体容量超过预设的限度！")

    def get_fitness(self, fitness_func, *args):
        """
        计算个体的适应度(这个暂时还有点问题)
        Args:
            fitness_func: 计算适应度值的函数名
            *args: 计算适应度所需要的变量，用元组来传递
        """
        fitness = fitness_func(*args)
        return fitness

    def update_fitness(self, id, new_fitness):
        """
        更新适应度
        Args:
            id: 需要修改适应度的个体id
            new_fitness: 新的适应度值
        """
        find_flag = 0  # 0表示未找到
        for individual in self.group:
            if individual.id == id:
                individual.fitness = new_fitness
                find_flag = 1
                return

        if find_flag == 0:
            print(f"群体中并没有id为{id}的个体!")

    def roulette_wheel_selection(self, selection_num):
        """
        轮盘赌选择
        Args:
            selection_num: 选择的个体数量
        """
        num_population = len(self.group)
        selection_flag = [0 for _ in range(num_population)]
        fitness_list = [individual.fitness for individual in self.group]
        total_fitness = sum(fitness_list)

        selection_probs = [sum(fitness_list[:i + 1]) / total_fitness for i in range(num_population)]

        selected_num = 0
        max_test = 1000
        while selected_num < selection_num and max_test > 0:
            random_prob = random.random()
            for j in range(num_population):
                if random_prob <= selection_probs[j] and selection_flag[j] != 1:
                    selection_flag[j] = 1  # 表示被选中
                    selected_num += 1
                    break
            max_test -= 1

        if max_test == 0:
            print("检测失败")
            return False

        # 删除后的群体
        new_group = []
        update_group = []
        for i in range(num_population):
            if selection_flag[i] == 1:
                new_group.append(self.group[i])
            else:
                update_group.append(self.group[i])
        self.group = new_group
        self.update_group = update_group
        print("检测成功")
        return True

    def tournament_selection(self, group_num, selection_num):
        """
        锦标赛选择,需要重写
        Args:
            group_num: 竞赛组规模
            selection_num: 选择的个体数量
        """
        num_population = len(self.group)
        selection_flag = [0 for _ in range(num_population)]

        selected_num = 0
        while selected_num < selection_num:
            sample_individual = random.sample(range(num_population), k=group_num)
            sample_fitness = [self.group[i].fitness for i in sample_individual]
            sample_index = sample_fitness.index(max(sample_fitness))
            max_index = sample_individual[sample_index]
            if selection_flag[max_index] == 0:
                selection_flag[max_index] = 1
                selected_num += 1

        # 删除后的群体
        new_group = []
        for i in range(num_population):
            if selection_flag[i] == 1:
                new_group.append(self.group[i])
        self.group = new_group

    def fitness_based_selection(self, selection_num):
        """
        基于适应度的选择（选择适应度排名前selection_num的个体保留）
        Args:
            selection_num: 选择的个体数目
        """
        sort_fitness = sorted(self.group, key=lambda g: g.fitness, reverse=True)
        self.group = sort_fitness[:selection_num]

        # 再按照id排序
        sort_id = sorted(self.group, key=lambda g: g.id, reverse=False)
        self.group = sort_id

    def single_point_crossover(self, parent_1, parent_2):
        """
        单点交叉
        Args:
            parent_1: 父亲节点1
            parent_2: 父亲节点2
        """
        chromosome_1 = parent_1.chromosome
        chromosome_2 = parent_2.chromosome
        crossover_point = random.randint(1, len(chromosome_1) - 1)
        offspring_1 = chromosome_1[:crossover_point] + chromosome_2[crossover_point:]
        offspring_2 = chromosome_2[:crossover_point] + chromosome_1[crossover_point:]

        child_1 = self.Individual(None, offspring_1, None)
        child_2 = self.Individual(None, offspring_2, None)
        # if len(self.update_group):
        #     self.update_group[self.update_group_id].chromosome = offspring_1
        #     self.update_group_id += 1
        # if len(self.update_group) > 1:
        #     self.update_group[self.update_group_id].chromosome = offspring_2
        #     self.update_group_id += 1
        return child_1, child_2

    def two_point_crossover(self, parent_1, parent_2):
        """
        两点交叉
        Args:
            parent_1: 父亲节点1
            parent_2: 父亲节点2
        """
        chromosome_1 = parent_1.chromosome
        chromosome_2 = parent_2.chromosome
        print(chromosome_1)
        print(chromosome_2)
        points = sorted(random.sample(range(1, len(chromosome_1)), 2))
        offspring_1 = chromosome_1[:points[0]] + chromosome_2[points[0]:points[1]] + chromosome_1[points[1]:]
        offspring_2 = chromosome_2[:points[0]] + chromosome_1[points[0]:points[1]] + chromosome_2[points[1]:]

        child_1 = self.Individual(None, offspring_1, None)
        child_2 = self.Individual(None, offspring_2, None)
        return child_1, child_2

    def multi_point_crossover(self, parent_1, parent_2, num_points=2):
        """
        多点交叉
        Args:
            parent_1: 父亲节点1
            parent_2: 父亲节点2
            num_points: 交叉点数
        """
        chromosome_1 = parent_1.chromosome
        chromosome_2 = parent_2.chromosome
        num_points = min(num_points, len(chromosome_1) - 1)
        points = sorted(random.sample(range(1, len(chromosome_1)), num_points))

        offspring_1 = chromosome_1
        offspring_2 = chromosome_2
        for point in points:
            offspring_1 = offspring_1[:point] + chromosome_2[point:]
            offspring_2 = offspring_2[:point] + chromosome_1[point:]

        child_1 = self.Individual(None, offspring_1, None)
        child_2 = self.Individual(None, offspring_2, None)
        return child_1, child_2

    def uniform_crossover(self, parent_1, parent_2, cross_prob):
        """
        均匀交叉
        Args:
            parent_1: 父亲节点1
            parent_2: 父亲节点2
            cross_prob: 交叉概率
        """
        chromosome_1 = parent_1.chromosome
        chromosome_2 = parent_2.chromosome
        offspring_1 = []
        offspring_2 = []
        len_chromosome = len(chromosome_1)

        for i in range(len_chromosome):
            if random.random() > cross_prob:
                # 不交叉
                offspring_1.append(chromosome_1[i])
                offspring_2.append(chromosome_2[i])
            else:
                # 交叉
                offspring_1.append(chromosome_2[i])
                offspring_2.append(chromosome_1[i])

        child_1 = self.Individual(None, offspring_1, None)
        child_2 = self.Individual(None, offspring_2, None)
        return child_1, child_2

    def reproduce_descendant(self, selection_input: tuple, crossover_input: tuple):
        """
        通过繁殖，使种群数量达到原始规模(新加入的方法)
        选择方法有：
            roulette_wheel_selection：轮盘赌选择
            tournament_selection： 锦标赛选择
            fitness_based_selection：基于适应度选择
        交叉方法有：
            single_point_crossover：单点交叉
            two_point_crossover：两点交叉
            multi_point_crossover：多点交叉
            uniform_crossover：均匀交叉
        Args:
            selection_input: 选择操作的参数，以元组形式传入，例：(方法名, 参数) -> (roulette_wheel_selection, 80), 如果有多个参数，则参数以列表形式传入
            crossover_input: 交叉操作参数，以元组形式传入，没有参数则参数一栏填写None, (single_point_crossover, None), 如果有多个参数，则参数以列表形式传入
        """
        print("-" * 30 + "开始产生后代" + "-" * 30)
        selection_method, selection_args = selection_input
        crossover_method, crossover_args = crossover_input

        self.update_info = []
        self.update_group = []
        self.update_group_id = 0

        if selection_method == "roulette_wheel_selection":
            if not isinstance(selection_args, int):
                print("参数有误！轮盘赌方法需要传入一个int类型的参数！")
                return
            ans = self.roulette_wheel_selection(selection_args)
            if not ans:
                print("失败")
                return
            print("-" * 15 + "roulette_wheel_selection..." + "-" * 15)

        elif selection_method == "tournament_selection":
            if not isinstance(selection_args, list) or len(selection_args) != 2:
                print("参数有误！锦标赛选择需要传入两个参数，请将两个参数以列表形式传入！")
                return
            arg_1 = selection_args[0]
            arg_2 = selection_args[1]
            self.tournament_selection(arg_1, arg_2)
            print("-" * 15 + "tournament_selection..." + "-" * 15)

        elif selection_method == "fitness_based_selection":
            if not isinstance(selection_args, int):
                print("参数有误！基于适应度方法需要传入一个int类型的参数！")
                return
            self.fitness_based_selection(selection_args)
            print("-" * 15 + "fitness_based_selection..." + "-" * 15)

        else:
            print("请输入正确的选择方法以及参数！")

        crossover_num = ceil((self.population_capacity - len(self.group))/2)
        if crossover_num == 0:
            print("种群数量已达到最大，不需要交叉操作来产生新的种群！")
            return

        # match crossover_method:
        if crossover_method == "single_point_crossover":
            if crossover_args is not None:
                print("单点交叉不需要任何参数！")
            for i in range(crossover_num):
                parent_1, parent_2 = random.sample(self.group, k=2)
                born_1, born_2 = self.single_point_crossover(parent_1, parent_2)
                self.update_info.append([born_1.id, born_1.chromosome, born_1.fitness])
                self.update_info.append([born_2.id, born_2.chromosome, born_2.fitness])

        elif crossover_method ==  "two_point_crossover":
            if crossover_args is not None:
                print("两点交叉不需要任何参数！")
            for i in range(crossover_num):
                parent_1, parent_2 = random.sample(self.group, k=2)
                born_1, born_2 = self.two_point_crossover(parent_1, parent_2)
                self.update_info.append([born_1.id, born_1.chromosome, born_1.fitness])
                self.update_info.append([born_2.id, born_2.chromosome, born_2.fitness])

        elif crossover_method == "multi_point_crossover":
            if crossover_args is None:
                crossover_args = 2
            elif not isinstance(crossover_args, int):
                print("多点交叉的参数为int类型，请输入正确的交叉参数！")
                return
            for i in range(crossover_num):
                parent_1, parent_2 = random.sample(self.group, k=2)
                born_1, born_2 = self.multi_point_crossover(parent_1, parent_2, crossover_args)
                self.update_info.append([born_1.id, born_1.chromosome, born_1.fitness])
                self.update_info.append([born_2.id, born_2.chromosome, born_2.fitness])

        elif crossover_method == "uniform_crossover":
            if crossover_args is None or crossover_args not in range(0, 1):
                print("均匀交叉需要一个(0, 1)之间的参数！")
                return
            for i in range(crossover_num):
                parent_1, parent_2 = random.sample(self.group, k=2)
                born_1, born_2 = self.uniform_crossover(parent_1, parent_2, crossover_args)
                self.update_info.append([born_1.id, born_1.chromosome, born_1.fitness])
                self.update_info.append([born_2.id, born_2.chromosome, born_2.fitness])
        else:
            print("请输入正确的交叉方法以及参数！")


# 模仿学习
# step1:选择邻居组成学习组织
# step2:确定能力属性
# step3:学习对象的识别
# step4:模仿学习
# step5:组织解散
class ImitationLearning:
    class Individual:
        def __init__(self, id: int, feature: list):
            """
            Args:
                id: 个体唯一的id
                feature: 个体特征属性
            """
            self.id = id
            self.money = random.randint(0, 1)
            self.feature = feature
            self.feature_effect = [1 for i in range(len(self.feature))]  # 能力属性效果，必须于feature中的顺序一致,默认全都为正向属性
            self.feature_normalize = []                    # 标准化后的属性值

    # 学习组织
    def __init__(self, learner: Individual, neighbors: List[Individual], imitation_rate):
        """
        Args:
            learner: 学习者,类型限定为Individual
            neighbors: 邻居,类型限定为list,list中的每个元素都是Individual类型。可以是随机邻居、冯诺依曼邻居、摩尔邻居以及复杂网络邻居
            imitation_rate: 模仿概率。并不是所有属性都会完全模仿
        """
        self.learner = learner
        self.neighbors = neighbors
        self.imitation_rate = imitation_rate

    # 确定能力属性,正向能力属性->1,负向能力属性->-1
    # 属性在评定个体优秀程度时的重要程度
    def set_feature_effect(self, effect):
        self.learner.feature_effect = effect

        for neighbor in self.neighbors:
            neighbor.feature_effect = effect

    # 能力属性标准化
    def feature_normalize(self):
        len_feature = len(self.learner.feature)

        # 找出每种能力属性的最大值以及最小值
        feature_max = []
        feature_min = []
        for i in range(len_feature):
            tmp_feature = [self.learner.feature[i]]
            for neighbor in self.neighbors:
                tmp_feature.append(neighbor.feature[i])

            if max(tmp_feature) == min(tmp_feature):  # 当最大值和最小值一样时
                feature_max.append(1)
                feature_min.append(1)
            else:
                feature_max.append(max(tmp_feature))
                feature_min.append(min(tmp_feature))

        # 对每个个体的能力属性进行标准化
        for i in range(len_feature):
            # 个体能力属性标准化
            if self.learner.feature_effect[i] == 1:
                self.learner.feature_normalize.append(
                    (self.learner.feature[i] - feature_min[i]) / (feature_max[i] - feature_min[i])
                )
            else:
                self.learner.feature_normalize.append(
                    (feature_max[i] - self.learner.feature[i]) / (feature_max[i] - feature_min[i])
                )

            # 邻居能力属性标准化
            len_neighbors = len(self.neighbors)
            for j in range(len_neighbors):
                if self.neighbors[j].feature_effect[i] == 1:
                    self.neighbors[j].feature_normalize.append(
                        (self.neighbors[j].feature[i] - feature_min[i]) / (feature_max[i] - feature_min[i])
                    )
                else:
                    self.neighbors[j].feature_normalize.append(
                        (feature_max[i] - self.neighbors[j].feature[i]) / (feature_max[i] - feature_min[i])
                    )

    # 比较组织中最佳部分
    def compare_best(self):
        len_feature = len(self.learner.feature)
        len_neighbors = len(self.neighbors)
        # 拥有最佳属性的邻居索引
        best_index = [-1 for i in range(len_feature)]       # -1表示自身

        for i in range(len_feature):
            for j in range(len_neighbors):
                if self.neighbors[j].feature_normalize[i] == 1:
                    best_index[i] = j

        return best_index

    # 综合优秀特征生成虚拟被模仿者，进而完成模仿学习
    def group_imitation(self):
        self.feature_normalize()        # 标准化
        best_index = self.compare_best()        # 获得拥有最佳属性邻居的索引
        len_feature = len(self.learner.feature)

        for i in range(len_feature):
            idx = best_index[i]
            if idx != -1:
                best_feature = self.neighbors[idx].feature[i]
                imit_prob = random.random()
                if imit_prob <= self.imitation_rate:
                    # print('*'*15 + "imitating..." + '*'*15)
                    self.learner.feature[i] = best_feature

    def neighbors_score_rider(self):
        #  获得邻居的score: money
        scores = []
        for rider in self.neighbors:
            scores.append(rider.agent_calculate_score(rider.w1, rider.w2, rider.w3))
        return scores

    # 计算个体得分
    def calculate_score(self):
        len_feature = len(self.learner.feature)
        len_neighbors = len(self.neighbors)
        # neighbors_score = [0 for i in range(len_neighbors)]
        neighbors_score = self.neighbors_score_rider()
        learner_score = self.learner.agent_calculate_score(self.learner.w1, self.learner.w2, self.learner.w3)

        for i in range(len_feature):
            # learner_score = learner_score + self.learner.feature[i]*self.learner.feature_effect[i]
            learner_score = learner_score
            for j in range(len_neighbors):
                # neighbors_score[j] = neighbors_score[j] + self.neighbors[j].feature[i]*self.neighbors[
                # j].feature_effect[i]
                neighbors_score[j] = neighbors_score[j]

        # 控制小数位数在两位以内
        learner_score = round(learner_score, 2)
        for j in range(len_neighbors):
            neighbors_score[j] = round(neighbors_score[j], 2)
        print("获得分数", learner_score, neighbors_score)
        return learner_score, neighbors_score

    # 向组织中综合性能最优秀的个体进行学习
    def individual_imitation(self):
        len_feature = len(self.learner.feature)
        learner_score, neighbors_score = self.calculate_score()
        max_score = max(neighbors_score)
        max_index = neighbors_score.index(max_score)
        best_feature = self.neighbors[max_index].feature
        print("特征学习", best_feature)

        if max_score > learner_score:
            imit_prob = random.random()
            if imit_prob <= self.imitation_rate:
                for i in range(len_feature):
                    self.learner.feature[i] = best_feature[i]
            return max_index
        return -1

class QLearning:
  def __init__(self,
               lr = 1e-3,
               discount_rate = 0.999,
               epsilon = 0.1):
      self.grid_values = defaultdict(float)
      self.lr = lr
      self.discount_rate = discount_rate
      self.epsilon = epsilon  # 探索率

  def get_grid_value(self, pos) -> float:
      if random.random() > self.epsilon:
          return self.grid_values[str(pos)]
      else: # 探索
          return 5 * random.random()
  
  def get_ture_grid_value(self, pos) -> float:
      return self.grid_values[str(pos)]

  def update_value(self, pos, delta) -> None:
      self.grid_values[str(pos)]+= delta
  
  def learn(self,pos, start_pt, end_pt, reward, duration):
      gamma = math.pow(self.discount_rate, duration)
      v0 = self.get_ture_grid_value(start_pt)
      v1 = self.get_ture_grid_value(end_pt)
      delta = reward + gamma * v1 - v0
      self.update_value(pos, delta)




# 强化学习 Q Learning
class QLEarning:
    def __init__(self):
        # 环境参数
        self.reduce_ration = 20         # 地图缩小倍率
        self.n_locations = 600          # * 暂定20*30=600，地图缩小20倍，通过具体的数据修改
        self.n_time_segments = 3        # 一天分为三个时间段
        self.n_order_density = 1        # * 订单密集程度分为三个级别，低，中，高（暂时未用到，设置为1，若是后面需要用到则设置为相应的数字）
        self.n_rider_states = 2         # 骑手的状态：送单/游走，休息保持不动，不算在观察值中
        self.n_time_steps = 2           # 一个时间步代表两分钟

        # 行动空间
        self.actions = ["up", "down", "left", "right", "static"]

        # 状态空间大小
        self.n_states = self.n_locations * self.n_time_segments * self.n_order_density * self.n_rider_states
        self.n_actions = len(self.actions)

        # 初始化Q表
        self.Q = np.zeros((self.n_states, self.n_actions))

        # 超参数
        self.alpha = 0.1  # 学习率
        self.gamma = 0.6  # 折扣因子
        self.epsilon = 0.1  # 探索率

    # 状态编码：将位置、时间段、订单密集程度和骑手状态的组合编码为一个唯一的状态ID
    def encode_state(self, x, y, time_segment, order_density, rider_state):
        x = int(x / self.reduce_ration)
        y = int(y / self.reduce_ration)
        code = (x * y * time_segment * order_density * rider_state +
                y * time_segment * order_density * rider_state +
                time_segment * order_density * rider_state +
                order_density * rider_state +
                rider_state)
        return int(code)

    # 状态编码解码
    def decode_state(self, state_code):
        rider_state = state_code % 10
        state_code = int(state_code / 10)

        order_density = state_code % 10
        state_code = int(state_code / 10)

        time_segment = state_code % 10
        state_code = int(state_code / 10)

        y = state_code % 100
        state_code = int(state_code / 100)

        x = state_code
        return [x, y, time_segment, order_density, rider_state]

    # 奖励函数
    def get_reward(self, old_state, new_state):
        # 奖励函数和现在的状态，采取的行动以及下一个状态有关
        old_state = self.decode_state(old_state)
        new_state = self.decode_state(new_state)

        density_change = new_state[3] - old_state[3]
        state_change = new_state[4] - old_state[4]

        # 计算reward（根据训练结果修改）
        action_reward = density_change * 1
        if state_change == 0:
            if new_state[4] == 0:  # 空驶
                action_reward += -1
            else:  # 带着订单移动
                action_reward += 1
        else:
            if state_change == 1:  # 接到订单
                action_reward += 5
            else:  # 完成订单
                action_reward += 1

        return action_reward

    # 选择动作, 训练时用不到，决策时需要用到
    # * 使用时需要判断给出的action是否有效
    def choose_action(self, current_state):
        if random.uniform(0, 1) < self.epsilon:
            return random.choice(range(self.n_actions))  # 探索
        else:
            return np.argmax(self.Q[current_state, :])  # 利用（选择Q值最高的动作）

    # 把输入的坐标字符串转换为列表
    def get_position(self, ps):
        position = [0, 0]
        flag = 0  # 0表示对x操作，1表示对y操作
        for s in ps:
            if s.isdigit():
                s = int(s)
                position[flag] = position[flag] * 10 + s
            else:
                flag = 1
        return position

    # 获得时间段
    def get_time_segment(self, step):
        minutes = int(60/self.n_time_segments)
        # 0代表0:00-8:00
        # 1代表8:00-16:00
        # 2代表16:00-24:00
        if step in range(0, 8*minutes):
            return 0
        elif step in range(8*minutes, 16*minutes):
            return 1
        else:
            return 2

    # 获得状态
    def get_status(self, current_state):
        # 0表示休息，1表示游走，2表示派单
        if current_state == 'a':
            return 0
        elif current_state == 'b':
            return 1
        else:
            return 2

    # 训练Q表
    def training(self, file_path):
        """
        Args:
            file_path: 训练数据文件的存储地址
        """
        # start_time = time.time()  # 记录程序运行时间
        # 读取文件中记录的信息
        df = pd.read_csv(file_path)
        df_data = df.values
        insert_data = np.array(df_data[len(df_data) - 1])       # 添加一行防止索引越界
        df_data = np.vstack((df_data, insert_data))

        episode_num = len(df_data)

        for episode in range(episode_num-1):
            # print(f"-" * 20 + "第%d个episode" % episode + "-" * 20)
            x, y = self.get_position(df_data[episode][2])
            time_segment = self.get_time_segment(df_data[episode][1])
            state = self.get_status(df_data[episode][3])
            if state == 0:
                continue
            current_state = self.encode_state(
                x, y, time_segment=time_segment, order_density=1, rider_state=state
            )
            # print(current_state)
            next_x, next_y = self.get_position(df_data[episode+1][2])
            next_time_segment = self.get_time_segment(df_data[episode+1][1])
            next_state = self.get_status(df_data[episode+1][3])
            next_state = self.encode_state(
                next_x, next_y, next_time_segment, 1, next_state
            )
            reward = self.get_reward(current_state, next_state)
            # 计算状态转移采取的动作
            action = 0
            delta_x = next_x - x
            delta_y = next_y - y
            # print("delta_x", delta_x)
            # print("delta_y", delta_y)
            if delta_y == self.reduce_ration:
                action = 0
            elif delta_y == -self.reduce_ration:
                action = 1
            elif delta_x == -self.reduce_ration:
                action = 2
            elif delta_x == self.reduce_ration:
                action = 3
            else:
                action = 4

            # Q表更新
            old_value = self.Q[current_state, action]
            next_max = np.max(self.Q[next_state, :])

            # 核心的Q学习更新公式
            self.Q[current_state, action] = old_value + self.alpha * (reward + self.gamma * next_max - old_value)
            # current_state = next_state
        # print("-*"*30)
        # print(self.Q)

        # df_qt = pd.DataFrame(self.Q, columns=["up", "down", "left", "right", "static"])
        # df_qt.to_csv("QTable.csv")

        # end_time = time.time()  # 记录程序运行时间
        # run_time = end_time - start_time
        # print("Running time:", run_time)


if __name__ == '__main__':
    print("-" * 20 + "Testing QLearning Module" + "-" * 20)
    ql = QLEarning()
    for i in range(10):
        ql.training(f"../agent_log/agent_{i}.csv")
    df_qt = pd.DataFrame(ql.Q, columns=["up", "down", "left", "right", "static"])
    df_qt.to_csv("QTable.csv")
    print(ql.Q)

    # 根据当前的状态查询q表采取动作
    # current_state = ql.encode_state(
    #     x=101, y=100, time_segment=ql.get_time_segment(56), order_density=1, rider_state=1
    # )
    # ql.choose_action(current_state)

    # print("测试模仿学习")
    #
    # feature_length = 10
    # learner = ImitationLearning.Individual(
    #     random.randint(0, 10), [random.randint(0, 100) for i in range(feature_length)]
    # )
    #
    # neighbors = []
    # for i in range(4):
    #     neighbors.append(
    #         ImitationLearning.Individual(
    #             random.randint(0, 10), [random.randint(0, 100) for j in range(feature_length)]
    #         )
    #     )
    #
    # il = ImitationLearning(learner, neighbors, 0.5)
    #
    # print("-"*30 + "learner属性值" + "-"*30)
    # print(il.learner.feature)
    # print("-" * 30 + "neighbors属性值" + "-" * 30)
    # for neighbor in il.neighbors:
    #     print(neighbor.feature)
    # print("-" * 30 + "属性值影响" + "-" * 30)
    # print(il.learner.feature_effect)
    #
    # feature_effect = [random.choice([-1, 1]) for i in range(feature_length)]
    # il.set_feature_effect(feature_effect)
    #
    # # il.group_imitation()
    #
    # print("——" * 30 + "模仿组织中优秀个体" + "——" * 30)
    # new_effect = [round(random.random(), 2) for i in range(feature_length)]
    # print("new_effect", new_effect)
    # il.set_feature_effect(new_effect)
    # il.individual_imitation()
    #
    # print("-" * 30 + "模仿后的learner属性值" + "-" * 30)
    # print(il.learner.feature)

    # feature_length = 10
    # learner = EvolutionaryLearning.Individual(
    #     random.randint(0, 10),
    #     [random.randint(0, 100) for i in range(feature_length)],
    #     random.randint(1, 10)
    # )
    #
    # neighbors = []
    # el = EvolutionaryLearning(10)
    # for i in range(6):
    #     el.add_to_group(
    #         i,
    #         [random.randint(0, 100) for j in range(feature_length)],
    #         random.randint(0, 10)
    #     )
    # for i in el.group:
    #     print("before", i.id)
    # el.roulette_wheel_selection(4)
    # parent_1, parent_2 = random.sample(el.group, k=2)
    # born_1, born_2 = el.single_point_crossover(parent_1, parent_2)
    # print("t",born_1.chromosome)
    # print(born_2.chromosome)
    # for i in el.group:
    #     print("after", i.id)

